
# Import libraries

# System libraries to import the data
import os.path 
from io import BytesIO
from urllib.request import urlopen
from zipfile import ZipFile

# Math libraries to process the data
import numpy as np
import pandas as pd

# Library for predictive data analysis
from sklearn import decomposition, preprocessing

# Graph libraries to produce graphs
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots

## If you use Notebook (and not JupyterLab), uncomment following lines
import plotly.io as pio
pio.renderers.default='notebook'


# Download ZIP and extract CSV
data_local_path = 'data/'
csv_filename = 'fr.openfoodfacts.org.products.csv'
csv_local_path = data_local_path+csv_filename

if not os.path.isfile(csv_local_path):
    # only if the file is not already present
    zip_filename = csv_filename+'.zip'
    zip_url = 'https://s3-eu-west-1.amazonaws.com/static.oc-static.com/prod/courses/files/parcours-data-scientist/P2/'+zip_filename
    zip_local_path = data_local_path+zip_filename

    with urlopen(zip_url) as zip_response:
        with ZipFile(BytesIO(zip_response.read())) as zip_file:
            # extract all files do local data/ directory
            zip_file.extractall(data_local_path)


clean_filename = 'fr.openfoodfacts.org.products-clean.csv'
clean_local_path = data_local_path+clean_filename

if not os.path.isfile(clean_local_path):
    # only if the clean file is not already presnt
    with open(csv_local_path, 'r') as csv_file, open(clean_local_path, 'w') as clean_file:
        """ Deal with irregularities

            23 data points are wrongly split into two lines : 
            - lines : 189070, 189105, 189111, 189121, 189154, 189162, 189164, 189170, 189244, 189246, 
                    189250, 189252, 189262, 189264, 189271, 189274, 189347, 189364, 189366, 189381, 
                    189406, 189408, 189419
            
            The pattern is always the same : 
            - a NewLine character (`\n`) is placed at the end of column "first_packaging_code_geo" 
            - and the next line starts with a TAB separator (`\t`) : column "cities" is empty.
            
            Since the first column ("code") is never empty, we just remove any `\n` character that is 
            directly followed by a TAB separator (`\t`).
        """

        data = csv_file.read()
        clean_file.write(data.replace('\n\t', '\t'))


# Read column names
column_names = pd.read_csv(clean_local_path, sep='\t', encoding='utf-8', nrows=0).columns.values

# Set column types according to fields description (https://static.openfoodfacts.org/data/data-fields.txt)
column_types = {col: 'Int64' for (col) in column_names if col.endswith(('_t', '_n'))}
column_types |= {col: float for (col) in column_names if col.endswith(('_100g', '_serving'))}
column_types |= {col: str for (col) in column_names if not col.endswith(('_t', '_n', '_100g', '_serving', '_tags'))}

tags_converter = lambda list_as_string_value : list_as_string_value.split(',') if list_as_string_value else pd.NA

# Load raw data
raw_data = pd.read_csv(clean_local_path, sep='\t', encoding='utf-8',
    dtype=column_types,
    parse_dates=[col for (col) in column_names if col.endswith('_datetime')],
    infer_datetime_format=True,
    converters={
        # Convert '_tags' columns into list of values (separator : ',')
        col: tags_converter
        for (col) in column_names if col.endswith('_tags')
    }
)

# Display DataFrame size
raw_data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 320749 entries, 0 to 320748
Columns: 162 entries, code to water-hardness_100g
dtypes: Int64(5), datetime64[ns](2), float64(99), object(56)
memory usage: 398.0+ MB


# Let's define a function to reuse this graph later
def plot_empty_values(dataframe: pd.DataFrame) -> None:
    """ Plot a histogram of empty values percentage per columns of the input DataFrame
    """
    num_rows = len(dataframe.index)
    columns_emptiness = pd.DataFrame({
        col : { 
            'count': dataframe[col].isna().sum(),
            'percent': 100 * dataframe[col].isna().sum() / num_rows,
        } for col in dataframe.columns
    }).transpose().sort_values(by=['count'])

    fig = px.bar(columns_emptiness,
        color='percent',
        y='percent',
        labels={
            'index':'column name',
            'percent':'% of empty values',
            'count':'# of empty values',
        },
        hover_data=['count'],
        title='Empty values per column',
        width=1200,
        height=600,
    )
    fig.show()

plot_empty_values(raw_data)


# Let's keep only meaningful columns
meaningful_columns = [
    # General information
    'code', 'product_name', 'main_category', 'additives_n', 

    # Nutri-Score
    'nutrition_grade_fr', 'nutrition-score-fr_100g',

    # Positive nutrition facts
    'energy_100g', 'saturated-fat_100g', 'sugars_100g', 'salt_100g',

    # Negative nutrition facts 
    'fruits-vegetables-nuts_100g', 'fiber_100g', 'proteins_100g',
]
meaningful_data = raw_data.loc[:, meaningful_columns].copy()

# Display DataFrame size
meaningful_data.info()

# Display first values of each column
meaningful_data.head()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 320749 entries, 0 to 320748
Data columns (total 13 columns):
 #   Column                       Non-Null Count   Dtype  
---  ------                       --------------   -----  
 0   code                         320749 non-null  object 
 1   product_name                 302987 non-null  object 
 2   main_category                84389 non-null   object 
 3   additives_n                  248961 non-null  Int64  
 4   nutrition_grade_fr           221233 non-null  object 
 5   nutrition-score-fr_100g      221233 non-null  float64
 6   energy_100g                  261136 non-null  float64
 7   saturated-fat_100g           229577 non-null  float64
 8   sugars_100g                  244994 non-null  float64
 9   salt_100g                    255533 non-null  float64
 10  fruits-vegetables-nuts_100g  3046 non-null    float64
 11  fiber_100g                   200891 non-null  float64
 12  proteins_100g                259929 non-null  float64
dtypes: Int64(1), float64(8), object(4)
memory usage: 32.1+ MB


# Display statistical summary of each column
meaningful_data.describe(include="all", datetime_is_numeric=True)


# Display the density of product categories
fig = px.line(raw_data['main_category'].value_counts())
fig.update_layout(
    title_text="Product categories",
    width=1200,
    height=800,
)
fig.show()


# Let's keep only the top values and merge the rest into "Other"
meaningful_data.loc[:,'top_category'] = raw_data['main_category'].where(
    raw_data['main_category'].isna() | raw_data['main_category'].isin(raw_data['main_category'].value_counts().index[:20]), 
    other='other', 
)

fig = make_subplots(
    rows=1, cols=2, 
    subplot_titles=("Top 20 with 'other'", "Top 20"), 
    specs=[[{'type':'domain'}, {'type':'domain'}]],
)
fig.add_trace(go.Pie(
    labels=meaningful_data['top_category'].value_counts().index, 
    values=meaningful_data['top_category'].value_counts().values, 
    name="Including 'other'",
    pull=[0.05 if cat == 'other' else 0 for cat in meaningful_data['top_category'].value_counts().index],
), row=1, col=1)
fig.add_trace(go.Pie(
    labels=meaningful_data['top_category'].value_counts().index[1:], 
    values=meaningful_data['top_category'].value_counts().values[1:], 
    name="Top 20",
), row=1, col=2)
fig.update_traces(
    textposition='inside',
    textinfo='percent+label'
)
fig.update_layout(
    title_text="Product categories",
    width=1200,
    height=600,
)
fig.show()


NUTRITION_GRADES = ('a', 'b', 'c', 'd', 'e')
TOP_CATEGORIES = tuple(meaningful_data['top_category'].value_counts().index)

# Display the nutrition grade distribution per product category
fig = px.histogram(meaningful_data.loc[meaningful_data['top_category'].notnull() & meaningful_data['nutrition_grade_fr'].notnull()],
    x='nutrition_grade_fr',
    category_orders={'nutrition_grade_fr': NUTRITION_GRADES, 'top_category': TOP_CATEGORIES},
    color='top_category',
    title='Global nutrition grade repartition by product category',
    width=1200,
    height=600,
)
fig.show()


# Display the product category distribution by nutrition grade
fig = px.histogram(meaningful_data.loc[meaningful_data['top_category'] != 'other'].loc[meaningful_data['top_category'].notnull() & meaningful_data['nutrition_grade_fr'].notnull()],
    x='top_category',
    category_orders={'nutrition_grade_fr': NUTRITION_GRADES, 'top_category': TOP_CATEGORIES},
    color='nutrition_grade_fr',
    title='Top 20 product categories repartition by nutrition grade',
    width=1200,
    height=600,
)
fig.show()


# Let's define a function that can be reused
def remove_negative_values(dataframe: pd.DataFrame, columns: list[str]) -> pd.DataFrame:
    """ Remove negative values from specified columns of DataFrame
    """
    df = dataframe.copy()
    for col in columns:
        df.loc[:,col] = dataframe[col].where(dataframe[col] >= 0)

    return df


positive_columns = [
    'additives_n', 
    'energy_100g', 'saturated-fat_100g', 'sugars_100g', 'salt_100g',
    'fruits-vegetables-nuts_100g', 'fiber_100g', 'proteins_100g',
]

clean_data = remove_negative_values(meaningful_data, positive_columns)

clean_data.describe()


# Let's define a function that can be reused
def remove_outliers(dataframe: pd.DataFrame, columns: list[str]) -> pd.DataFrame:
    """ Remove outlier values from specified columns of DataFrame

        Compute the Inter-Quartile Ranges and set outliers to NaN
    """
    df = dataframe.copy()

    # compute quartiles and define range
    quartiles = df[columns].quantile([0.25, 0.75])
    iqr = quartiles.loc[0.75]-quartiles.loc[0.25]
    limits = pd.DataFrame({
        col: [
            quartiles.loc[0.25, col] - 1.5 * iqr[col], # min
            quartiles.loc[0.75, col] + 1.5 * iqr[col], # max
        ] for col in columns
    }, index=['min', 'max'])

    # set to NaN data that are outside the range
    for col in columns:
        df.loc[:,col] = dataframe[col].where(
            limits.loc['min', col] <= dataframe[col]
        ).where(
            dataframe[col] <= limits.loc['max', col]
        )

    return df


numeric_columns = positive_columns.copy()
numeric_columns.append('nutrition-score-fr_100g')

clean_data = remove_outliers(clean_data, numeric_columns)

clean_data.describe()


# Let's define a function to plot multiple BoxPlots
def draw_boxplots(dataframe: pd.DataFrame, categorical_column: str, numerical_columns: list[str], order_values: tuple[str] = None, num_cols: int = 3) -> None:
    """ Draw one boxplot per numerical variable, split per categories.

        Arguments :
        - dataframe : Pandas DataFrame containing the data, including the categorical_column and numerical_columns
        - categorical_column : string representing the name of the variable containing the categories
        - numerical_columns : list of strings representing the name of the numerical variables to plot
        - order_values : list of strings representing the values of the numerical variables to plot

        Returns : None
    """
    num_lines = int(np.ceil(len(numerical_columns) / num_cols))
    fig, axes = plt.subplots(num_lines, num_cols, figsize=(8 * num_cols , 8 * num_lines))
    fig.suptitle(f'Numeric variables distribution, per { categorical_column }', fontsize=24)

    for i, col in enumerate(numerical_columns):
        sns.boxplot(data=dataframe,
            x=categorical_column, 
            y=col,
            order=order_values,
            showmeans=True,
            ax=axes[int(np.floor(i / num_cols)), i % num_cols],
        )

# Draw the BoxPlots of each numeric column, split per Nutrition Grade
draw_boxplots(
    dataframe=clean_data, 
    categorical_column='nutrition_grade_fr', 
    numerical_columns=numeric_columns, 
    order_values=NUTRITION_GRADES
)


# Let's work on a copy of our clean data
super_clean_data = clean_data.copy()

for grade in NUTRITION_GRADES:
    # for each nutrition grade, 
    # we remove the outliers of each numeric column detected 
    # after filtering the data by nutrition grade
    super_clean_data.loc[super_clean_data['nutrition_grade_fr'] == grade] = remove_outliers(clean_data.loc[clean_data['nutrition_grade_fr'] == grade], numeric_columns)

super_clean_data.describe()


# Let's draw the BoxPlots again after more data cleaning
draw_boxplots(
    dataframe=super_clean_data, 
    categorical_column='nutrition_grade_fr', 
    numerical_columns=numeric_columns, 
    order_values=NUTRITION_GRADES
)


# Let's count duplucated lines
duplicates = super_clean_data.duplicated()
duplicates.describe()

count     320749
unique         1
top        False
freq      320749
dtype: object


# Let's drop raws where nutrition_grade_fr is empty
nutrition_data = super_clean_data.dropna(subset=['nutrition_grade_fr']).copy()
plot_empty_values(nutrition_data)


# Let's compute the mean of each numeric variable
means = nutrition_data.groupby('nutrition_grade_fr').mean()

# Special processing for the number of additives, which has to be a round number
means.loc[:,'additives_n'] = means['additives_n'].map(np.round)

# Fill empty values with means
for grade in means.index:
    nutrition_data.loc[nutrition_data['nutrition_grade_fr'] == grade] = nutrition_data[nutrition_data['nutrition_grade_fr'] == grade].fillna(
        value=means.loc[grade]
    )

# Let's see if we still have empty values
plot_empty_values(nutrition_data)


# First, let's compute the contingency matrix, excluding category 'other'
contingency_table = nutrition_data.loc[nutrition_data['top_category'] != 'other'].pivot_table(
    values='code',
    index='top_category',
    columns='nutrition_grade_fr',
    aggfunc='count',
    fill_value=0,
    observed=True,
).sort_values(by=list(NUTRITION_GRADES))

# Let's compute the normalized contributions to dependency of each variable
tx = contingency_table.sum(axis='columns').to_frame()
ty = contingency_table.sum(axis='index').to_frame().T
indep = tx.dot(ty) / len(nutrition_data)
measure = (contingency_table - indep)**2 / indep
xi_n = measure.sum().sum()
table = measure / xi_n 

table.sort_values(
    by=list(NUTRITION_GRADES), 
    ascending=False,
    inplace=True, 
)

# Display the heatmap of top product categories per nutrition grade
fig = px.imshow(table,
    title="Normalised correlations between Nutrition Grades and Product Categories",
    width=1200,
    height=600,
)
fig.show()


# Let's draw the BoxPlots again after more data cleaning
draw_boxplots(
    dataframe=nutrition_data, 
    categorical_column='nutrition_grade_fr', 
    numerical_columns=numeric_columns, 
    order_values=NUTRITION_GRADES
)


# Let's define the correlation function
def eta_squared(x,y):
    """ Compute the correlation ratio between categorical variable (x) and numeric variable (y)
    """
    moyenne_y = y.mean()
    classes = []
    for classe in x.unique():
        yi_classe = y[x==classe]
        classes.append({'ni': len(yi_classe),
                        'moyenne_classe': yi_classe.mean()})
    SCT = sum([(yj-moyenne_y)**2 for yj in y])
    SCE = sum([c['ni']*(c['moyenne_classe']-moyenne_y)**2 for c in classes])
    return SCE/SCT

# Compute the E² value for each numeric variable
anova=pd.DataFrame(columns=['eta_squared'])
for col in numeric_columns:
    anova.loc[col] = eta_squared(nutrition_data['nutrition_grade_fr'],nutrition_data[col])

# Sort the variables : most influential first
sorted_anova = anova.sort_values(by='eta_squared', ascending=False)

# plot the graph
fig = px.scatter(sorted_anova,
    x=sorted_anova.index,
    y=sorted_anova['eta_squared'],
    color='eta_squared',
    size='eta_squared',
    title='Correlation between Nutri-Score grade and nutrition variables',
    width=1200,
    height=600,
)
fig.show()


# Let's keep only the top 3 values and merge the rest into "Other"
nutrition_data.loc[:,'top_top_category'] = nutrition_data['main_category'].where(
    nutrition_data['main_category'].isna() | nutrition_data['main_category'].isin(nutrition_data['main_category'].value_counts().index[:3]), 
    other='other', 
)

# Let's draw the BoxPlots again after more data cleaning
draw_boxplots(
    dataframe=nutrition_data, 
    categorical_column='top_top_category', 
    numerical_columns=numeric_columns, 
    order_values=None
)


fig = px.scatter_matrix(nutrition_data.sample(frac=.01),
    dimensions=[
        'energy_100g', 
        'saturated-fat_100g', 
        'sugars_100g', 
        'salt_100g', 
        'nutrition-score-fr_100g',
    ],
    color="nutrition_grade_fr",
    symbol="nutrition_grade_fr",
    category_orders={'nutrition_grade_fr': NUTRITION_GRADES},
    hover_data=['product_name', 'main_category'],
    opacity=.2,
    width=1200,
    height=1200,
    title="Distribution of products over variables increasing the Nutri-Score",
)
fig.update_traces(
    showupperhalf=False,
    diagonal_visible=False,
)
fig.show()


fig = px.scatter_matrix(nutrition_data.sample(frac=.01),
    dimensions=[
        'fruits-vegetables-nuts_100g', 
        'fiber_100g',
        'proteins_100g',
        'nutrition-score-fr_100g',
    ],
    color="nutrition_grade_fr", 
    symbol="nutrition_grade_fr",
    category_orders={'nutrition_grade_fr': NUTRITION_GRADES},
    hover_data=['product_name', 'main_category'],
    opacity=.2,
    width=1200,
    height=1200,
    title="Distribution of products over variables decreasing the Nutri-Score",
)
fig.update_traces(
    showupperhalf=False,
    diagonal_visible=False,
)
fig.show()


corr = nutrition_data[[
        'additives_n', 
        'energy_100g', 
        'saturated-fat_100g', 
        'sugars_100g', 
        'salt_100g', 
        # 'nutrition-score-fr_100g',
        'fruits-vegetables-nuts_100g', 
        'fiber_100g',
        'proteins_100g',
    ]].corr()

fig = px.imshow(corr.where(np.tril(np.ones(corr.shape), -1).astype(bool)),
    title="Pairwise Pearson correlations of nutrition variables",
    width=1200,
    height=600,
)
fig.show()


# Let's normalize each numeric value
normalized_nutrition_data = nutrition_data.copy()
normalized_nutrition_data[numeric_columns]=( nutrition_data[numeric_columns] - nutrition_data[numeric_columns].mean() ) / nutrition_data[numeric_columns].std()

# Plot 
fig = px.scatter(normalized_nutrition_data.sample(frac=.01),
    x=[
        'additives_n', 
        'energy_100g', 
        'saturated-fat_100g', 
        'sugars_100g', 
        'salt_100g', 
        'fruits-vegetables-nuts_100g', 
        'fiber_100g',
        'proteins_100g',
    ],
    y='nutrition-score-fr_100g',
    hover_data=['product_name', 'main_category'],
    trendline='ols', # Ordinary Least Squares
    opacity=.2,
    width=1200,
    height=600,
    title="Linear regression models of nutritive values",
)
fig.show()


# Let's choose 3 new components
num_components=3

# Let's instantiate our PCA proccessor
pca = decomposition.PCA(n_components=num_components)

# Let's compute the components and project our data onto the new frame defined by the components
projected_nutrition_data = pd.DataFrame(
    data=pca.fit_transform(normalized_nutrition_data[numeric_columns]), 
    index=normalized_nutrition_data.index, 
    columns=[ f'PC{i}' for i in range(1, num_components+1) ]
)
projected_nutrition_data[['nutrition_grade_fr', 'top_top_category']] = nutrition_data[['nutrition_grade_fr', 'top_top_category']]

# Let's plot a 3D Scatter graph of our data in the new frame of reference
fig = px.scatter_3d(projected_nutrition_data.sample(frac=.01),
    x=projected_nutrition_data.columns[0], 
    y=projected_nutrition_data.columns[1], 
    z=projected_nutrition_data.columns[2],
    color="nutrition_grade_fr",
    symbol="nutrition_grade_fr",
    category_orders={'nutrition_grade_fr': NUTRITION_GRADES},
    opacity=.2,
    width=1200,
    height=800,
    title="Products projection in the Principal Components frame",
)
fig.show()


# Let's compute the variance ratio of each component
pca_components_variance_ratio = pd.DataFrame(
    data=pca.explained_variance_ratio_,
    index=projected_nutrition_data.columns[:-2],
    columns=['variance'],
)

fig = px.scatter(pca_components_variance_ratio,
    x=pca_components_variance_ratio.index,
    y='variance',
    color=pca_components_variance_ratio['variance'],
    size=pca_components_variance_ratio['variance'],
    title='Variance ratio of each principal componants',
    width=1200,
    height=400,
)
fig.show()


fig = go.Figure()

fig.add_trace(go.Box(
    x=projected_nutrition_data['nutrition_grade_fr'],
    y=projected_nutrition_data['PC1'],
    name='PC1',
))

fig.add_trace(go.Box(
    x=projected_nutrition_data['nutrition_grade_fr'],
    y=projected_nutrition_data['PC2'],
    name='PC2',
))

fig.add_trace(go.Box(
    x=projected_nutrition_data['nutrition_grade_fr'],
    y=projected_nutrition_data['PC3'],
    name='PC3',
))

fig.update_layout(
    xaxis={
        'categoryorder': 'array',
        'categoryarray': NUTRITION_GRADES,
    },
    xaxis_title='Nutri-Score grade',
    yaxis_title='Products repartition along Principal Components, per Nutri-Score grade',
    boxmode='group',
    width=1200,
    height=800,
)

fig.show()


fig = go.Figure()

fig.add_trace(go.Box(
    x=projected_nutrition_data['top_top_category'],
    y=projected_nutrition_data['PC1'],
    name='PC1',
))

fig.add_trace(go.Box(
    x=projected_nutrition_data['top_top_category'],
    y=projected_nutrition_data['PC2'],
    name='PC2',
))

fig.add_trace(go.Box(
    x=projected_nutrition_data['top_top_category'],
    y=projected_nutrition_data['PC3'],
    name='PC3',
))

fig.update_layout(
    xaxis_title='Product Category',
    yaxis_title='Products repartition along Principal Components, per Product Category',
    boxmode='group',
    width=1200,
    height=800,
)

fig.show()


# Let's decompose each component in the initial variables frame
pca_components=pd.DataFrame(
    data=pca.components_,
    index=projected_nutrition_data.columns[:-2],
    columns=numeric_columns,
)

pca_components.sort_values(
    by=list(pca_components.index), 
    axis='columns',
    ascending=False,
    inplace=True, 
)

fig = px.imshow(pca_components,
    title="Contribution of numeric variable to each principal Component",
    width=1200,
    height=600,
)
fig.show()

	code	product_name	main_category	additives_n	nutrition_grade_fr	nutrition-score-fr_100g	energy_100g	saturated-fat_100g	sugars_100g	salt_100g	fruits-vegetables-nuts_100g	fiber_100g	proteins_100g
count	320749	302987	84389	248961.000000	221233	221233.000000	2.611360e+05	229577.000000	244994.000000	255533.000000	3046.000000	200891.000000	259929.000000
unique	320749	221343	3543	NaN	5	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
top	8410218167365	Ice Cream	en:beverages	NaN	d	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
freq	1	410	6054	NaN	62763	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
mean	NaN	NaN	NaN	1.936384	NaN	9.164736	1.141842e+03	5.129562	16.002983	2.028455	31.376615	2.862048	7.076074
std	NaN	NaN	NaN	2.502319	NaN	9.055796	6.446875e+03	8.013929	22.326319	128.263683	31.947699	12.867424	8.408824
min	NaN	NaN	NaN	0.000000	NaN	-15.000000	0.000000e+00	0.000000	-17.860000	0.000000	0.000000	-6.700000	-800.000000
25%	NaN	NaN	NaN	0.000000	NaN	1.000000	3.770000e+02	0.000000	1.300000	0.063500	0.000000	0.000000	0.700000
50%	NaN	NaN	NaN	1.000000	NaN	10.000000	1.100000e+03	1.790000	5.710000	0.581660	22.100000	1.500000	4.760000
75%	NaN	NaN	NaN	3.000000	NaN	16.000000	1.674000e+03	7.140000	24.000000	1.374140	50.950000	3.600000	10.000000
max	NaN	NaN	NaN	31.000000	NaN	40.000000	3.251373e+06	550.000000	3520.000000	64312.800000	100.000000	5380.000000	430.000000

	additives_n	nutrition-score-fr_100g	energy_100g	saturated-fat_100g	sugars_100g	salt_100g	fruits-vegetables-nuts_100g	fiber_100g	proteins_100g
count	248961.000000	221233.000000	2.611360e+05	229577.000000	244987.000000	255533.000000	3046.000000	200890.000000	259926.000000
mean	1.936384	9.164736	1.141842e+03	5.129562	16.003589	2.028455	31.376615	2.862096	7.081171
std	2.502319	9.055796	6.446875e+03	8.013929	22.326329	128.263683	31.947699	12.867438	8.198381
min	0.000000	-15.000000	0.000000e+00	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
25%	0.000000	1.000000	3.770000e+02	0.000000	1.300000	0.063500	0.000000	0.000000	0.700000
50%	1.000000	10.000000	1.100000e+03	1.790000	5.710000	0.581660	22.100000	1.500000	4.760000
75%	3.000000	16.000000	1.674000e+03	7.140000	24.000000	1.374140	50.950000	3.600000	10.000000
max	31.000000	40.000000	3.251373e+06	550.000000	3520.000000	64312.800000	100.000000	5380.000000	430.000000

	additives_n	nutrition-score-fr_100g	energy_100g	saturated-fat_100g	sugars_100g	salt_100g	fruits-vegetables-nuts_100g	fiber_100g	proteins_100g
count	239154.000000	221229.000000	260056.000000	212330.000000	229679.000000	239046.000000	3046.000000	186763.000000	247336.000000
mean	1.601332	9.164178	1114.283467	3.502375	12.130071	0.718235	31.376615	1.914724	5.887857
std	1.832342	9.054928	781.784025	4.548240	15.161208	0.774207	31.947699	2.213743	5.998511
min	0.000000	-15.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
25%	0.000000	1.000000	372.000000	0.000000	1.080000	0.048260	0.000000	0.000000	0.500000
50%	1.000000	10.000000	1094.000000	1.400000	4.800000	0.474980	22.100000	1.200000	4.230000
75%	3.000000	16.000000	1674.000000	5.700000	18.400000	1.186180	50.950000	3.300000	9.000000
max	7.000000	38.000000	3619.000000	17.840000	58.000000	3.340000	100.000000	9.000000	23.940000

	additives_n	nutrition-score-fr_100g	energy_100g	saturated-fat_100g	sugars_100g	salt_100g	fruits-vegetables-nuts_100g	fiber_100g	proteins_100g
count	233835.000000	219602.000000	254958.000000	206037.000000	224249.000000	238747.000000	3043.000000	182606.000000	240982.000000
mean	1.525738	9.152203	1106.116924	3.497238	11.791241	0.716328	31.308962	1.789232	5.506563
std	1.766967	9.003385	770.379808	4.587804	14.999885	0.772293	31.890646	2.068080	5.579444
min	0.000000	-9.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
25%	0.000000	1.000000	379.000000	0.000000	1.000000	0.046200	0.000000	0.000000	0.500000
50%	1.000000	10.000000	1084.000000	1.200000	4.500000	0.470000	22.000000	1.200000	4.000000
75%	2.000000	16.000000	1661.000000	5.810000	17.650000	1.181100	50.550000	3.000000	8.400000
max	7.000000	30.000000	3615.000000	17.810000	58.000000	3.337560	100.000000	9.000000	23.940000

Santé Publique France : rendre les données de santé publique plus accessibles¶

Contexte¶

Outils utilisés¶

Chargement des données et premier aperçu¶

Téléchargement et extraction des données¶

Gestion des irrégularités du fichier CSV téléchargé¶

Chargement des données¶

Sélection des données pertinentes¶

Premier aperçu¶

Première analyse statistique¶

Distribution des différentes valeurs¶

Variable catégorique nominale : `main_category`¶

Variable catégorique ordinale : `nutrition_grade_fr`¶

Variables numérique¶

Nettoyage : suppression des données aberrantes¶

Nettoyage : suppression des doublons et lignes vides¶

Nettoyage : remplacement des valeurs manquantes¶

Recherche et analyse des corrélations¶

Corrélation entre catégories de produits et Nutri-Score¶

Corrélation entre valeurs nutritives et note de Nutri-Score¶

Corrélation entre valeurs nutritives et catégories de produits¶

Corrélation entre les différentes valeurs nutritives¶

Modélisation de l'influence des valeurs nutritives sur le Nutri-Score¶

	code	product_name	main_category	additives_n	nutrition_grade_fr	nutrition-score-fr_100g	energy_100g	saturated-fat_100g	sugars_100g	salt_100g	fruits-vegetables-nuts_100g	fiber_100g	proteins_100g
0	0000000003087	Farine de blé noir	NaN	<NA>	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN
1	0000000004530	Banana Chips Sweetened (Whole)	NaN	0	d	14.0	2243.0	28.57	14.29	0.00000	NaN	3.6	3.57
2	0000000004559	Peanuts	NaN	0	b	0.0	1941.0	0.00	17.86	0.63500	NaN	7.1	17.86
3	0000000016087	Organic Salted Nut Mix	NaN	0	d	12.0	2540.0	5.36	3.57	1.22428	NaN	7.1	17.86
4	0000000016094	Organic Polenta	NaN	0	NaN	NaN	1552.0	NaN	NaN	NaN	NaN	5.7	8.57

Santé Publique France : rendre les données de santé publique plus accessibles¶

Contexte¶

Outils utilisés¶

Chargement des données et premier aperçu¶

Téléchargement et extraction des données¶

Gestion des irrégularités du fichier CSV téléchargé¶

Chargement des données¶

Sélection des données pertinentes¶

Premier aperçu¶

Première analyse statistique¶

Distribution des différentes valeurs¶

Variable catégorique nominale : main_category¶

Variable catégorique ordinale : nutrition_grade_fr¶

Variables numérique¶

Nettoyage : suppression des données aberrantes¶

Nettoyage : suppression des doublons et lignes vides¶

Nettoyage : remplacement des valeurs manquantes¶

Recherche et analyse des corrélations¶

Corrélation entre catégories de produits et Nutri-Score¶

Corrélation entre valeurs nutritives et note de Nutri-Score¶

Corrélation entre valeurs nutritives et catégories de produits¶

Corrélation entre les différentes valeurs nutritives¶

Modélisation de l'influence des valeurs nutritives sur le Nutri-Score¶

Variable catégorique nominale : `main_category`¶

Variable catégorique ordinale : `nutrition_grade_fr`¶